superstrict

import "../strutil/strutil.bmx"
import "SourceLine.bmx"

type Token

	'// -------------------------------------------------------------------------------------------------------------
	'// Token types: Non-keywords.
	'// -------------------------------------------------------------------------------------------------------------
	
	'// Special tokens.
	const TYPE_NONE:short = 0;
	const TYPE_EOF:short = 1;
	
	'// Literals.
	const TYPE_INTEGER:short = 2;
	const TYPE_REAL:short = 6;
	const TYPE_STRING:short = 7;
	const TYPE_CHAR:short = 8;
	
	'// Identifier.
	const TYPE_IDENT:short = 20;
	
	'// Punctuators.
	const TYPE_OPEN_BRACE:short = 100;
	const TYPE_CLOSE_BRACE:short = 101;
	const TYPE_OPEN_PAREN:short = 102;
	const TYPE_CLOSE_PAREN:short = 103;
	const TYPE_OPEN_SQUARE:short = 104;
	const TYPE_CLOSE_SQUARE:short = 105;
	const TYPE_PLUS:short = 106;
	const TYPE_MINUS:short = 107;
	const TYPE_ASTERISK:short = 108;
	const TYPE_DIVIDE:short = 109;
	const TYPE_SIGN:short = 110;
	const TYPE_ASSIGN:short = 111;
	const TYPE_AMPERSAND:short = 112;
	const TYPE_VBAR:short = 113;
	const TYPE_OPEN_ANGEL:short = 114;
	const TYPE_CLOSE_ANGEL:short = 115;
	const TYPE_COMMA:short = 116;
	const TYPE_DOT:short = 117;
	const TYPE_COLON:short = 118;
	const TYPE_PERCENT:short = 119;
	const TYPE_CARET:short = 120;
	const TYPE_QUESTION:short = 121;
	const TYPE_SEMICOLON:short = 122;
	
	'// -------------------------------------------------------------------------------------------------------------
	'// Token types: Reserved identifiers.
	'// -------------------------------------------------------------------------------------------------------------
	
	'// Datatypes.
	const RID_BOOL:short = 300;
	const RID_CHAR:short = 301;
	const RID_INT:short = 302;
	const RID_REAL:short = 303;
	const RID_STRING:short = 304;
	
	'// Literals.
	const RID_TRUE:short = 350;
	const RID_FALSE:short = 351;
	const RID_NULL:short = 352;
	
	'// Declarators.
	const RID_DATABASE:short = 400;
	const RID_TABLE:short = 401;
	const RID_ROW:short = 402;
	
	'// -------------------------------------------------------------------------------------------------------------
	'// Create, setup and return a new token.
	'// -------------------------------------------------------------------------------------------------------------
	function create:Token(line:SourceLine, _type:short, lexeme:string)
	
		local t:Token = new Token;
		
		t.m_line = line;
		t.m_type = _type;
		t.m_lexeme = lexeme;
		
		return t;
	
	end function
	
	'// -------------------------------------------------------------------------------------------------------------
	'// Poke a character onto the lexeme.
	'// -------------------------------------------------------------------------------------------------------------
	method poke(char:byte)
	
		m_lexeme :+ chr(char);
	
	end method
	
	'// -------------------------------------------------------------------------------------------------------------
	'// Convert the token type to a string and return it.
	'// -------------------------------------------------------------------------------------------------------------
	method typeToString:string()
	
		select (m_type)
		
			case TYPE_NONE
				return "null";
			case TYPE_EOF
				return "eof";
			case TYPE_OPEN_BRACE, ..
			     TYPE_CLOSE_BRACE, ..
			     TYPE_OPEN_PAREN, ..
			     TYPE_CLOSE_PAREN, ..
			     TYPE_OPEN_SQUARE, ..
			     TYPE_CLOSE_SQUARE, ..
			     TYPE_PLUS, ..
			     TYPE_MINUS, ..
			     TYPE_ASTERISK, ..
			     TYPE_DIVIDE, ..
			     TYPE_SIGN, ..
			     TYPE_ASSIGN, ..
			     TYPE_AMPERSAND, ..
			     TYPE_VBAR, ..
			     TYPE_AMPERSAND, ..
			     TYPE_OPEN_ANGEL, ..
			     TYPE_CLOSE_ANGEL, ..
			     TYPE_COMMA, ..
			     TYPE_DOT, ..
			     TYPE_COLON, ..
			     TYPE_PERCENT, ..
			     TYPE_CARET, ..
			     TYPE_QUESTION
				return "punct";
			case TYPE_SEMICOLON
				return "end_column";
			case TYPE_IDENT
				return "ident";
			case TYPE_REAL
				return "real";
			case TYPE_STRING
				return "string";
			case TYPE_CHAR
				return "char";
			case RID_TRUE, ..
			     RID_FALSE
				return "bool";
			case RID_BOOL, ..
			     RID_INT, ..
			     RID_REAL, ..
			     RID_CHAR, ..
			     RID_STRING
				return "datatype";
			case RID_DATABASE, ..
			     RID_TABLE, ..
			     RID_ROW
				return "declarator";
		
		end select
	
	end method
	
	'// -------------------------------------------------------------------------------------------------------------
	'// Return the token as a string.
	'// -------------------------------------------------------------------------------------------------------------
	method toString:string()
	
		return "<" + typeToString() + " : " + m_lexeme + ">";
	
	end method
	
	'// -------------------------------------------------------------------------------------------------------------
	'// Return a line of info about this token, for use when dumping the token stream to disc.
	'// -------------------------------------------------------------------------------------------------------------
	method tokenInfo:string()
	
		local header:string;
		
		if (m_line) ..
			header = m_line.m_originPath + "(" + m_line.m_lineNumber; ..
		else ..
			header = "unknownFile(unknownLine";
		 
		header :+ "): " + typeToString();
		
		if (is_punctuator) ..
			header :+ " (" + m_type + ")";
		
		if (header.length < 80) ..
			header :+ strutil.repeatstr(" ", 80 - header.length);
		
		if (m_type = Token.TYPE_STRING) ..
			header :+ "~q" + m_lexeme + "~q"; ..
		else if (m_type = Token.TYPE_CHAR) ..
			header :+ "'" + m_lexeme + "'"; ..
		else ..
			header :+ m_lexeme;
		
		return header;
	
	end method
	
	'// -------------------------------------------------------------------------------------------------------------
	'// A pointer to the line in which this token is contained.
	'// -------------------------------------------------------------------------------------------------------------
	field m_line:SourceLine;
	
	'// -------------------------------------------------------------------------------------------------------------
	'// This determines what type of token this is.
	'// -------------------------------------------------------------------------------------------------------------
	field m_type:short;
	
	'// -------------------------------------------------------------------------------------------------------------
	'// The lexeme that this token represents, if any.
	'// -------------------------------------------------------------------------------------------------------------
	field m_lexeme:string;
	
	'// -------------------------------------------------------------------------------------------------------------
	'// Flags describing the token.
	'// -------------------------------------------------------------------------------------------------------------
	field is_literal:byte = false;
	field is_punctuator:byte = false;
	field is_identifier:byte = false;
	field is_keyword:byte = false;

end type
